{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import sys\n",
    "\n",
    "SOURCE_DIR = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__name__))))\n",
    "sys.path.insert(0, SOURCE_DIR)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import tensorflow as tf\n",
    "from malaya.train.model.pegasus import transformer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "# tf.enable_eager_execution()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From /Users/huseinzolkepli/Documents/malaya/malaya/train/model/pegasus/layers/attention.py:46: The name tf.layers.Dense is deprecated. Please use tf.compat.v1.layers.Dense instead.\n",
      "\n",
      "WARNING:tensorflow:From /Users/huseinzolkepli/Documents/malaya/malaya/train/model/pegasus/layers/embedding.py:61: The name tf.variable_scope is deprecated. Please use tf.compat.v1.variable_scope instead.\n",
      "\n",
      "WARNING:tensorflow:From /Users/huseinzolkepli/Documents/malaya/malaya/train/model/pegasus/layers/embedding.py:61: The name tf.AUTO_REUSE is deprecated. Please use tf.compat.v1.AUTO_REUSE instead.\n",
      "\n",
      "WARNING:tensorflow:From /Users/huseinzolkepli/Documents/malaya/malaya/train/model/pegasus/layers/embedding.py:65: The name tf.get_variable is deprecated. Please use tf.compat.v1.get_variable instead.\n",
      "\n",
      "WARNING:tensorflow:From /Users/huseinzolkepli/Documents/malaya/malaya/train/model/pegasus/layers/embedding.py:69: calling RandomNormal.__init__ (from tensorflow.python.ops.init_ops) with dtype is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Call initializer instance with the dtype argument instead of passing it to the constructor\n",
      "WARNING:tensorflow:From /Users/huseinzolkepli/Documents/malaya/malaya/train/model/pegasus/layers/attention.py:131: The name tf.matrix_band_part is deprecated. Please use tf.linalg.band_part instead.\n",
      "\n",
      "WARNING:tensorflow:From /Users/huseinzolkepli/Documents/malaya/malaya/train/model/pegasus/transformer.py:140: The name tf.losses.softmax_cross_entropy is deprecated. Please use tf.compat.v1.losses.softmax_cross_entropy instead.\n",
      "\n",
      "WARNING:tensorflow:From /Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/site-packages/tensorflow_core/python/ops/losses/losses_impl.py:121: where (from tensorflow.python.ops.array_ops) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Use tf.where in 2.0, which has the same broadcast rule as np.where\n"
     ]
    },
    {
     "ename": "TypeError",
     "evalue": "pred must not be a Python bool",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mTypeError\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-4-1772483d5f93>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m     29\u001b[0m   {\n\u001b[1;32m     30\u001b[0m       \u001b[0;34m\"inputs\"\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mX\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 31\u001b[0;31m   }, tf.shape(X)[1] + 10, 1, top_p = 0.8, temperature = 0.5)\n\u001b[0m",
      "\u001b[0;32m~/Documents/malaya/malaya/train/model/pegasus/transformer.py\u001b[0m in \u001b[0;36mpredict\u001b[0;34m(self, features, max_decode_len, beam_size, **beam_kwargs)\u001b[0m\n\u001b[1;32m    193\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    194\u001b[0m         decodes_BxT = decoding.left2right_decode(\n\u001b[0;32m--> 195\u001b[0;31m             \u001b[0msymbols_to_logits_fn\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcache\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mB\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mT\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mV\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mbeam_size\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mbeam_kwargs\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    196\u001b[0m         )\n\u001b[1;32m    197\u001b[0m         \u001b[0;32mreturn\u001b[0m \u001b[0;34m{\u001b[0m\u001b[0;34m'outputs'\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mdecodes_BxT\u001b[0m\u001b[0;34m}\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/Documents/malaya/malaya/train/model/pegasus/layers/decoding.py\u001b[0m in \u001b[0;36mleft2right_decode\u001b[0;34m(symbols_to_logits_fn, context_BxU_dict, batch_size, max_decode_len, vocab_size, beam_size, beam_start, beam_alpha, beam_min, beam_max, temperature, top_k, top_p, eos_id)\u001b[0m\n\u001b[1;32m    195\u001b[0m             \u001b[0mloop_cond\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    196\u001b[0m             \u001b[0mdecode_loop\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 197\u001b[0;31m             \u001b[0;34m[\u001b[0m\u001b[0mtf\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mconstant\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdtype\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mdtype\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0minit_dec_BxT\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcontext_BxU_dict\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    198\u001b[0m         )\n\u001b[1;32m    199\u001b[0m         \u001b[0;32mreturn\u001b[0m \u001b[0mdecodes\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/site-packages/tensorflow_core/python/ops/control_flow_ops.py\u001b[0m in \u001b[0;36mwhile_loop\u001b[0;34m(cond, body, loop_vars, shape_invariants, parallel_iterations, back_prop, swap_memory, name, maximum_iterations, return_same_structure)\u001b[0m\n\u001b[1;32m   2751\u001b[0m       \u001b[0mops\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0madd_to_collection\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mops\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mGraphKeys\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mWHILE_CONTEXT\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mloop_context\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   2752\u001b[0m     result = loop_context.BuildLoop(cond, body, loop_vars, shape_invariants,\n\u001b[0;32m-> 2753\u001b[0;31m                                     return_same_structure)\n\u001b[0m\u001b[1;32m   2754\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0mmaximum_iterations\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   2755\u001b[0m       \u001b[0;32mreturn\u001b[0m \u001b[0mresult\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/site-packages/tensorflow_core/python/ops/control_flow_ops.py\u001b[0m in \u001b[0;36mBuildLoop\u001b[0;34m(self, pred, body, loop_vars, shape_invariants, return_same_structure)\u001b[0m\n\u001b[1;32m   2243\u001b[0m       \u001b[0;32mwith\u001b[0m \u001b[0mops\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mget_default_graph\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_mutation_lock\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m  \u001b[0;31m# pylint: disable=protected-access\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   2244\u001b[0m         original_body_result, exit_vars = self._BuildLoop(\n\u001b[0;32m-> 2245\u001b[0;31m             pred, body, original_loop_vars, loop_vars, shape_invariants)\n\u001b[0m\u001b[1;32m   2246\u001b[0m     \u001b[0;32mfinally\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   2247\u001b[0m       \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mExit\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/site-packages/tensorflow_core/python/ops/control_flow_ops.py\u001b[0m in \u001b[0;36m_BuildLoop\u001b[0;34m(self, pred, body, original_loop_vars, loop_vars, shape_invariants)\u001b[0m\n\u001b[1;32m   2168\u001b[0m         expand_composites=True)\n\u001b[1;32m   2169\u001b[0m     \u001b[0mpre_summaries\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mops\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mget_collection\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mops\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mGraphKeys\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_SUMMARY_COLLECTION\u001b[0m\u001b[0;34m)\u001b[0m  \u001b[0;31m# pylint: disable=protected-access\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 2170\u001b[0;31m     \u001b[0mbody_result\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mbody\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0mpacked_vars_for_body\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   2171\u001b[0m     \u001b[0mpost_summaries\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mops\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mget_collection\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mops\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mGraphKeys\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_SUMMARY_COLLECTION\u001b[0m\u001b[0;34m)\u001b[0m  \u001b[0;31m# pylint: disable=protected-access\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   2172\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0mnest\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mis_sequence_or_composite\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mbody_result\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/Documents/malaya/malaya/train/model/pegasus/layers/decoding.py\u001b[0m in \u001b[0;36mdecode_loop\u001b[0;34m(i, decodes_BxT, cache_BxU_dict)\u001b[0m\n\u001b[1;32m    175\u001b[0m         \u001b[0;32mdef\u001b[0m \u001b[0mdecode_loop\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mi\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdecodes_BxT\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcache_BxU_dict\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    176\u001b[0m             \u001b[0mlogits_BxV\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0msymbols_to_logits_fn\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdecodes_BxT\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcache_BxU_dict\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mi\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 177\u001b[0;31m             \u001b[0mlogits_BxV\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mprocess_logits\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mlogits_BxV\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtop_k\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtop_p\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtemperature\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    178\u001b[0m             decodes_BxT = inplace_update_i(\n\u001b[1;32m    179\u001b[0m                 \u001b[0mdecodes_BxT\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtf\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0margmax\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mlogits_BxV\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m-\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mi\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/Documents/malaya/malaya/train/model/pegasus/layers/decoding.py\u001b[0m in \u001b[0;36mprocess_logits\u001b[0;34m(logits_BxN, top_k, top_p, temperature)\u001b[0m\n\u001b[1;32m     89\u001b[0m         \u001b[0mtop_p\u001b[0m \u001b[0;34m>\u001b[0m \u001b[0;36m0\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     90\u001b[0m         \u001b[0;32mlambda\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mnucleus_sampling\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mlogits_BxN\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtop_p\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 91\u001b[0;31m         \u001b[0;32mlambda\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mlogits_BxN\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     92\u001b[0m     )\n\u001b[1;32m     93\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/site-packages/tensorflow_core/python/util/deprecation.py\u001b[0m in \u001b[0;36mnew_func\u001b[0;34m(*args, **kwargs)\u001b[0m\n\u001b[1;32m    505\u001b[0m                 \u001b[0;34m'in a future version'\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mdate\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mNone\u001b[0m \u001b[0;32melse\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0;34m'after %s'\u001b[0m \u001b[0;34m%\u001b[0m \u001b[0mdate\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    506\u001b[0m                 instructions)\n\u001b[0;32m--> 507\u001b[0;31m       \u001b[0;32mreturn\u001b[0m \u001b[0mfunc\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    508\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    509\u001b[0m     doc = _add_deprecated_arg_notice_to_docstring(\n",
      "\u001b[0;32m/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/site-packages/tensorflow_core/python/ops/control_flow_ops.py\u001b[0m in \u001b[0;36mcond\u001b[0;34m(pred, true_fn, false_fn, strict, name, fn1, fn2)\u001b[0m\n\u001b[1;32m   1209\u001b[0m     \u001b[0;31m# Add the Switch to the graph.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1210\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0misinstance\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mpred\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mbool\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1211\u001b[0;31m       \u001b[0;32mraise\u001b[0m \u001b[0mTypeError\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"pred must not be a Python bool\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1212\u001b[0m     \u001b[0mp_2\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mp_1\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mswitch\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mpred\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mpred\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1213\u001b[0m     \u001b[0mpivot_1\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0marray_ops\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0midentity\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mp_1\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mname\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m\"switch_t\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mTypeError\u001b[0m: pred must not be a Python bool"
     ]
    }
   ],
   "source": [
    "vocab_size = 12\n",
    "hidden_size = 16\n",
    "filter_size = 16\n",
    "num_encoder_layers = 2\n",
    "num_decoder_layers = 2\n",
    "num_heads = 2\n",
    "label_smoothing = 0.1\n",
    "dropout = 0.1\n",
    "beam_size = 3\n",
    "model = transformer.TransformerEncoderDecoderModel(vocab_size, hidden_size,\n",
    "                                                   filter_size, num_heads,\n",
    "                                                   num_encoder_layers,\n",
    "                                                   num_decoder_layers,\n",
    "                                                   label_smoothing, dropout)\n",
    "\n",
    "X = tf.placeholder(tf.int64, (None, None))\n",
    "Y = tf.placeholder(tf.int64, (None, None))\n",
    "\n",
    "# X = tf.ones((2, 7), tf.int64)\n",
    "# Y = tf.ones((2, 5), tf.int64)\n",
    "\n",
    "loss, outputs = model(\n",
    "    {\n",
    "        \"inputs\": X,\n",
    "        \"targets\": Y\n",
    "    }, True)\n",
    "\n",
    "outputs = model.predict(\n",
    "  {\n",
    "      \"inputs\": X,\n",
    "  }, tf.shape(X)[1] + 10, 1, top_p = 0.8, temperature = 0.5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(<tf.Tensor 'softmax_cross_entropy_loss/value:0' shape=() dtype=float32>,\n",
       " {'logits': <tf.Tensor 'Reshape_5:0' shape=(?, ?, 12) dtype=float32>})"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "loss, outputs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "sess = tf.InteractiveSession()\n",
    "sess.run(tf.global_variables_initializer())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "\n",
    "l, o = sess.run([loss, outputs], feed_dict = {X: np.ones((2, 7)), Y: np.ones((2, 5))})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'logits': array([[[ 0.16811992, -0.16834208,  0.36941448, -0.819937  ,\n",
       "           0.81210244,  3.048914  ,  0.3620631 , -1.154437  ,\n",
       "          -0.4472818 ,  1.2497824 , -1.1724902 ,  0.6941405 ],\n",
       "         [ 0.2713595 ,  1.3699167 , -0.66439694,  0.5983454 ,\n",
       "           0.45273322,  3.387432  ,  0.32195398, -0.69869095,\n",
       "          -0.21832518,  0.42171213, -0.40606117, -0.7956446 ],\n",
       "         [ 0.12774722,  1.3988615 , -0.82430494,  0.7635339 ,\n",
       "           0.43681467,  3.3545623 ,  0.31199497, -0.7435752 ,\n",
       "          -0.23870769,  0.12706661, -0.42601156, -1.0791947 ],\n",
       "         [ 0.00839269,  1.476044  , -1.0462202 ,  0.68440825,\n",
       "           0.49082965,  3.148295  ,  0.24370927, -0.5045557 ,\n",
       "          -0.37599453,  0.0331544 , -0.5250994 , -0.81289506],\n",
       "         [-0.13912864,  0.8509864 , -1.1274807 ,  0.05773274,\n",
       "           0.66053927,  3.1115117 ,  0.09243412, -0.41702795,\n",
       "          -0.3727442 ,  0.38563582, -0.56638855, -0.37897238]],\n",
       " \n",
       "        [[ 0.3872224 ,  0.3137273 , -0.30726355,  0.10818754,\n",
       "           0.43429974,  1.5299404 , -0.8514208 , -0.917902  ,\n",
       "          -0.7093521 ,  1.5025527 , -1.2484318 , -0.3984149 ],\n",
       "         [ 0.24686486,  1.1241944 , -0.5634689 ,  0.7523905 ,\n",
       "           0.5361041 ,  2.880868  , -0.30408126, -0.3584975 ,\n",
       "          -0.5974206 ,  1.0151753 , -0.78577   , -0.91615903],\n",
       "         [ 0.08810472,  1.1877792 , -0.47379187,  0.8360836 ,\n",
       "           0.58759063,  2.9353642 , -0.12068997, -0.4199066 ,\n",
       "          -0.60623556,  0.61870074, -0.78286815, -1.0685477 ],\n",
       "         [-0.27932888,  1.2408412 , -0.45041844,  0.8294446 ,\n",
       "           0.6008011 ,  2.6828485 ,  0.08158559, -0.4007096 ,\n",
       "          -0.6884948 ,  0.12352753, -0.88319725, -1.1785674 ],\n",
       "         [ 0.0689393 ,  1.0220996 , -0.5558489 ,  0.47139752,\n",
       "           0.826181  ,  2.7215285 , -0.19470075, -0.20474887,\n",
       "          -0.65647936,  0.5754616 , -0.8645274 , -0.8086908 ]]],\n",
       "       dtype=float32)}"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "o"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "2.5708184"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "l"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.7"
  },
  "varInspector": {
   "cols": {
    "lenName": 16,
    "lenType": 16,
    "lenVar": 40
   },
   "kernels_config": {
    "python": {
     "delete_cmd_postfix": "",
     "delete_cmd_prefix": "del ",
     "library": "var_list.py",
     "varRefreshCmd": "print(var_dic_list())"
    },
    "r": {
     "delete_cmd_postfix": ") ",
     "delete_cmd_prefix": "rm(",
     "library": "var_list.r",
     "varRefreshCmd": "cat(var_dic_list()) "
    }
   },
   "types_to_exclude": [
    "module",
    "function",
    "builtin_function_or_method",
    "instance",
    "_Feature"
   ],
   "window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
